/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package sinaEx;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import spider.Config;
import weibo4j.Friendships;
import weibo4j.Timeline;
import weibo4j.Users;
import weibo4j.model.Paging;
import weibo4j.model.Status;
import weibo4j.model.User;
import weibo4j.model.WeiboException;

/**
 *
 * @author Administrator
 */
public final class dataCollector {

    public static int time = 0;
    public static List<String> userid = new ArrayList<String>();
    public static Map<String, String> username = new HashMap<String, String>();
    public static Map<String, String> ctag = new HashMap<String, String>();
    public static Map<String, Boolean> verified = new HashMap<String, Boolean>();
    public static Map<String, Boolean> fame = new HashMap<String, Boolean>();
    public static Map<String, Integer> friendsCount = new HashMap<String, Integer>();
    public static Map<String, Integer> fansCount = new HashMap<String, Integer>();
    public static Map<String, Double> blogFrequency = new HashMap<String, Double>();
    public static Map<String, Map<String, Integer>> interestTag = new HashMap<String, Map<String, Integer>>();
    public static Map<String, Integer> mediaCount = new HashMap<String, Integer>();
    public static Map<String, Integer> agencyCount = new HashMap<String, Integer>();
    public static Map<String, Integer> celebrityCount = new HashMap<String, Integer>();
    public static Map<String, Integer> normalCount = new HashMap<String, Integer>();
    public static Map<String, Integer> websiteCount = new HashMap<String, Integer>();
    public static Map<String, Integer> brandCount = new HashMap<String, Integer>();
    public static Map<String, Integer> campusCount = new HashMap<String, Integer>();
    public static Map<String, Integer> zoombieCount = new HashMap<String, Integer>();
    public static Map<String, Integer> zoombieBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> campusBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> brandBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> websiteBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> normalBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> celebrityBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> agencyBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> mediaBlogCount = new HashMap<String, Integer>();
    public static Map<String, Integer> createBlogCount = new HashMap<String, Integer>();
    public static Map<String, String[]> celebrityType = new HashMap<String, String[]>();
    public static List<String> media = new ArrayList<String>();
    public static List<String> brand = new ArrayList<String>();
    public static List<String> agency = new ArrayList<String>();
    public static List<String> website = new ArrayList<String>();
    public static List<String> campus = new ArrayList<String>();
    public static List<String> celebrity = new ArrayList<String>();
    public static List<String> zoombie = new ArrayList<String>();
    public static List<String> pickup = new ArrayList<String>();

    dataCollector() throws IOException, WeiboException, InterruptedException {
        readCelebrity();
        readZoombie();
       String uid="";
        /*
        int i = 0;
       
        for (String c : celebrity) {
        if(i<200){
        i+=1;
        continue;
        }
        Users um = new Users();
        User userinfo = null;
        try {
        userinfo = um.showUserById(c);
        time += 1;
        } catch (Exception e) {
        int flag = 0;
        while (userinfo == null) {
        if (flag == 0) {
        System.out.println("time out!");
        }
        flag = 1;
        if (time > 10000) {
        Thread.sleep(300000);
        System.out.println("sleeping");
        }
        try {
        userinfo = um.showUserById(c);
        } catch (Exception ee) {
        }
        
        if (userinfo != null) {
        time = 1;
        }
        }
        }
        if (userinfo != null) {
        if (userinfo.getFriendsCount() < 50 || userinfo.getStatusesCount() < 50) {
        continue;
        }
        }
        time += 1;
        userid.add(c);
        getFriendsCountTag(c);
        getBlogFrequency(c);
        writeData(c);
        clear();
        i += 1;
        System.out.println(i);
        if (i >600) {
        break;
        }
        }
         */

        
        userid.add(uid);
        getFriendsCountTag(uid);
        getBlogFrequency(uid);
        writeData(uid);
        clear();
        int size = userid.size();
        int i = 1;
        System.out.println("Time: " + i + ". Current size is: " + (size));
        /*
        while (size < 50000) {
            List<String> pickuptemp = new ArrayList<String>();
            pickuptemp.addAll(pickup);
            pickup.clear();
            for (String id : pickuptemp) {
                if (userid.contains(id)) {
                    continue;
                }
                getFriendsCountTag(id);
                getBlogFrequency(id);
                writeData(id);
                clear();
                size = userid.size();
                System.out.println("Time: " + i + ". Current size is: " + (size) + " " + id);
                i += 1;
            }
        }
         * 
         */

    }

    public void readZoombie() throws IOException {
        String filename = "src/sinaEx/zoombie";
        BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
        String line = reader.readLine();
        while (line != null) {
            if (!line.isEmpty()) {
                String s[] = line.split(" ");
                zoombie.add(s[0]);
            }
            line = reader.readLine();
        }
        reader.close();
    }

    public void readCelebrity() throws IOException {
        String filename = "src/sinaEx/celebrity";
        BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
        String line = reader.readLine();
        line = reader.readLine();
        while (line != null) {
            String s[] = line.split("\t");
            String id = s[1];
            String type[] = {s[2], s[3], s[4]};
            celebrityType.put(s[1], type);
            if (s[4].equals("2")) {
                media.add(id);
            }
            if (s[4].equals("1")) {
                celebrity.add(id);
            }
            if (s[4].equals("3")) {
                brand.add(id);
            }
            if (s[4].equals("4") || s[4].equals("7")) {
                agency.add(id);
            }
            if (s[4].equals("5")) {
                website.add(id);
            }
            if (s[4].equals("6")) {
                campus.add(id);
            }
            celebrityType.put(id, type);
            line = reader.readLine();
        }
        reader.close();
    }

    //Get Tag of a celebrity
    public void getCelebrityTag(String uid, String id) {
        Map<String, Integer> tagCount = new HashMap<String, Integer>();
        int count = 0;
        if (interestTag.containsKey(uid)) {
            tagCount = interestTag.get(uid);
        }
        String key = celebrityType.get(id)[1];
        if (tagCount.containsKey(key)) {
            count = tagCount.get(key);
        }
        count += 1;
        tagCount.put(key, count);
        interestTag.put(uid, tagCount);
    }

    //Get a user;
    public void selectNextUsers(String uid) {
    }

    //Get all the information of a user;
    public void getFriendsCountTag(String uid) throws WeiboException, InterruptedException {

        int friendscount = basicInfo(uid);
        getCount(uid, friendscount);
    }

    //Basic info like uid, verified, friends count, fans count and username;
    public int basicInfo(String uid) throws WeiboException, InterruptedException {
        userid.add(uid);
        Users um = new Users();
        User userinfo = null;
        try {
            userinfo = um.showUserById(uid);
            time += 1;
        } catch (Exception e) {
            int flag = 0;
            while (userinfo == null) {
                if (flag == 0) {
                    System.out.println("time out!");
                }
                flag = 1;
                if (time > 10000) {
                    Thread.sleep(300000);
                    System.out.println("sleeping");
                }
                try {
                    userinfo = um.showUserById(uid);
                } catch (Exception ee) {
                }

                if (userinfo != null) {
                    time = 1;
                }
            }
        }
        time += 1;
        boolean v = userinfo.isVerified();
        if (celebrity.contains(uid)) {
            fame.put(uid, true);
        } else {
            fame.put(uid, false);
        }
        int friendscount = userinfo.getFriendsCount();
        int fanscount = userinfo.getFollowersCount();
        String uname = userinfo.getName();
        username.put(uid, uname);
        verified.put(uid, v);
        friendsCount.put(uid, friendscount);
        fansCount.put(uid, fanscount);
        return friendscount;
    }

    //All kinds of count;
    public void getCount(String uid, int friendscount) throws WeiboException, InterruptedException {
        Friendships fm = new Friendships();
        List<User> users = new ArrayList<User>();
        int page = (int) Math.ceil((double) friendscount / (double) 200);
        for (int i = 0; i < page; i++) {
            List<User> temp = null;
            try {
                temp = fm.getFriendsByID(uid, 200, i * 200);
                time += 1;
            } catch (Exception e) {
                int flag = 0;
                while (temp == null) {
                    if (flag == 0) {
                        System.out.println("time out!");
                    }
                    flag = 1;
                    if (time > 10000) {
                        Thread.sleep(300000);
                        System.out.println("sleeping");
                    }
                    try {
                        temp = fm.getFriendsByID(uid, 200, i * 200);
                    } catch (Exception ee) {
                    }
                    if (temp != null) {
                        time = 1;
                    }
                }
            }

            users.addAll(temp);
        }
        int mediacount = 0;
        int agencycount = 0;
        int zoombiecount = 0;
        int campuscount = 0;
        int websitecount = 0;
        int brandcount = 0;
        int celebritycount = 0;
        int normalcount = 0;
        for (User u : users) {
            String id = u.getId();
            int flag = 0;
            if (media.contains(id)) {
                flag = 1;
                mediacount += 1;
                mediaCount.put(uid, mediacount);
            }
            if (agency.contains(id)) {
                flag = 1;
                agencycount += 1;
                agencyCount.put(uid, agencycount);
            }
            if (campus.contains(id)) {
                flag = 1;
                campuscount += 1;
                campusCount.put(uid, campuscount);
            }
            if (website.contains(id)) {
                flag = 1;
                websitecount += 1;
                websiteCount.put(uid, websitecount);
            }
            if (brand.contains(id)) {
                flag = 1;
                brandcount += 1;
                brandCount.put(uid, brandcount);
            }
            if (zoombie.contains(id)) {
                flag = 1;
                zoombiecount += 1;
                zoombieCount.put(uid, zoombiecount);
            }
            if (celebrity.contains(id)) {
                flag = 1;
                int fan = u.getFavouritesCount();
                int submit = u.getStatusesCount();
                if (pickup.size() < 11) {
                    if (fan > 50 && submit > 100) {
                        if (!userid.contains(id)) {
                            pickup.add(id);
                        }
                    }
                }
                celebritycount += 1;
                getCelebrityTag(uid, id);
                celebrityCount.put(uid, celebritycount);
            }
            if (flag == 0) {
                int fan = u.getFavouritesCount();
                int submit = u.getStatusesCount();
                /*
                if (pickup.size() < 11) {
                if (fan > 50 && submit > 100) {
                if (!userid.contains(id)) {
                pickup.add(id);
                }
                }
                }
                 * */

                normalcount += 1;
                normalCount.put(uid, normalcount);
            }
        }
    }

    public void getBlogFrequency(String uid) throws WeiboException, InterruptedException {
        Timeline tm = new Timeline();
        List<Status> status = null;
        try {
            status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);
            time += 1;
        } catch (Exception e) {
            while (status == null) {
                int flag = 0;
                if (time > 10000) {
                    if (flag == 0) {
                        System.out.println("time out!");
                    }
                    flag = 1;
                    Thread.sleep(300000);
                    System.out.println("sleeping");
                }
                try {
                    status = tm.getUserTimeline(uid, username.get(uid), 200, new Paging(1, 200), 0, 0);
                } catch (Exception ee) {
                }

                if (status != null) {
                    time = 1;
                }
            }
        }
        time += 1;
        int sSize = status.size();
        if (sSize > 0) {
            long latestTime = status.get(0).getCreatedAt().getTime();
            long oldestTime = status.get(sSize - 1).getCreatedAt().getTime();
            double microblogFrequency = (double) sSize / ((latestTime - oldestTime) / 1000 / 60 / 60 / 24);
            blogFrequency.put(uid, microblogFrequency);
        } else {
            double microblogFrequency = 0;
            blogFrequency.put(uid, microblogFrequency);
        }

        int createcount = 0;
        int mediacount = 0;
        int agencycount = 0;
        int campuscount = 0;
        int brandcount = 0;
        int websitecount = 0;
        int celebritycount = 0;
        int normalcount = 0;
        int zoombiecount = 0;
        for (Status s : status) {
            if (s.getRetweetedStatus() != null) {
                Status os = s.getRetweetedStatus();
                if (os.getUser() != null) {
                    String id = os.getUser().getId();
                    if (media.contains(id)) {
                        if (mediaBlogCount.containsKey(id)) {
                            mediacount = mediaBlogCount.get(id);
                        }
                        mediacount += 1;
                        mediaBlogCount.put(uid, mediacount);
                    } else if (agency.contains(id)) {
                        agencycount += 1;
                        agencyBlogCount.put(uid, agencycount);
                    } else if (campus.contains(id)) {
                        campuscount += 1;
                        campusBlogCount.put(uid, campuscount);
                    } else if (website.contains(id)) {
                        websitecount += 1;
                        websiteBlogCount.put(uid, websitecount);
                    } else if (brand.contains(id)) {
                        brandcount += 1;
                        brandBlogCount.put(uid, brandcount);
                    } else if (zoombie.contains(id)) {
                        zoombiecount += 1;
                        zoombieBlogCount.put(uid, zoombiecount);
                    } else if (celebrity.contains(id)) {
                        celebritycount += 1;
                        celebrityBlogCount.put(uid, celebritycount);
                    } else {
                        if (!id.equals(uid)) {
                            normalcount += 1;
                            normalBlogCount.put(uid, normalcount);
                        }
                    }
                }
            } else {
                createcount += 1;
                createBlogCount.put(uid, createcount);
            }

        }
    }

    public void writeData(String id) {
        String interest = "";
        Map<String, Integer> tagCount = interestTag.get(id);
        if (tagCount != null) {
            Set keys = tagCount.keySet();
            int count = keys.size();

            for (int i = 0; i < count; i++) {
                String key = (String) keys.toArray()[i];
                int c = tagCount.get(key);
                interest += (key + "|" + c + "||");
            }
        }

        String query = "INSERT INTO `weibonews`.`wb_ex_network` (`uid`, `username`, `friendsCount`, `fansCount`, `fame`,`verified`, `interestTag`, `mediaCount`, `agencyCount`, `celebrityCount`, `normalCount`, `websiteCount`, `brandCount`, `campusCount`,`zoombieCount`,`blogFrequency`) VALUES ("
                + id + ","
                + "'" + username.get(id) + "',"
                + friendsCount.get(id) + ","
                + fansCount.get(id) + ","
                + fame.get(id) + ","
                + verified.get(id) + ","
                + "'" + interest + "'" + ","
                + (mediaCount.get(id)) + ","
                + (agencyCount.get(id)) + ","
                + (celebrityCount.get(id)) + ","
                + (normalCount.get(id)) + ","
                + (websiteCount.get(id)) + ","
                + (brandCount.get(id)) + ","
                + (campusCount.get(id)) + ","
                + (zoombieCount.get(id)) + ","
                + blogFrequency.get(id)
                + ");";
        String query2 = "INSERT INTO `weibonews`.`wb_ex_blogrepost` (`uid`, `agencyRC`, `mediaRC`, `zoombieRC`, `normalRC`, `campusRC`,  `websiteRC`, `celebrityRC`, `create`) VALUES ("
                + id + ","
                + agencyBlogCount.get(id) + ","
                + mediaBlogCount.get(id) + ","
                + zoombieBlogCount.get(id) + ","
                + normalBlogCount.get(id) + ","
                + campusBlogCount.get(id) + ","
                + websiteBlogCount.get(id) + ","
                + celebrityBlogCount.get(id) + ","
                + createBlogCount.get(id)
                + ");";
        String delete = "DELETE FROM `weibonews`.`wb_ex_network` WHERE `uid` not like 'A%';"
                + "DELETE FROM `weibonews`.`wb_ex_blogrepost` WHERE `uid` not like 'A%';";
        try {
            Class.forName("com.mysql.jdbc.Driver").newInstance();
            Connection conn = (Connection) DriverManager.getConnection("jdbc:mysql://localhost:3306/weibonews?useUnicode=true&characterEncoding=gb2312", "root", "");
            Statement stmt = (Statement) conn.createStatement();
            /*
            ResultSet result = (ResultSet) stmt.executeQuery("select * from `weibonews`.`wb_ex_network` where `uid` = '" + id + "';");
            if (result.next()) {
                System.out.println(result.first());
                System.out.println("returned");
                return;
            }
             * 
             */

            // stmt.executeUpdate(delete);
            stmt.executeUpdate(query);
            conn.close();
            Connection conn2 = (Connection) DriverManager.getConnection("jdbc:mysql://localhost:3306/weibonews?useUnicode=true&characterEncoding=gb2312", "root", "");
            Statement stmt2 = (Statement) conn2.createStatement();
            stmt2.executeUpdate(query2);
            conn2.close();
        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("Already Exist!");
        }
    }

    private void clear() {
        agencyBlogCount.clear();
        agencyCount.clear();
        blogFrequency.clear();
        brandBlogCount.clear();
        brandCount.clear();
        campusBlogCount.clear();
        campusCount.clear();
        celebrityBlogCount.clear();
        celebrityCount.clear();
        createBlogCount.clear();
        fansCount.clear();
        friendsCount.clear();
        interestTag.clear();
        mediaBlogCount.clear();
        mediaCount.clear();
        normalBlogCount.clear();
        normalCount.clear();
        username.clear();
        verified.clear();
        websiteBlogCount.clear();
        websiteCount.clear();
        zoombieBlogCount.clear();
        zoombieCount.clear();
    }
}
